package io.github.biezhi.elves.spider;

import java.net.InetSocketAddress;
import java.net.Proxy;
import java.net.SocketTimeoutException;
import java.util.ArrayList;
import java.util.List;
import java.util.ListIterator;
import java.util.Map;

import org.jsoup.Jsoup;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

import com.alibaba.fastjson.JSONObject;

import io.github.biezhi.elves.qcc.model.CompBasic;
import io.github.biezhi.elves.service.QccService;
import io.github.biezhi.elves.thread.QccRunnable;
import io.github.biezhi.elves.utils.StringUtils;

 /**
  * 爬取企查查
  * @author Administrator
  *
  */
@Component
public class QccSpider {

	@Autowired
	private QccService qccService;
	
	//第一批
	String qccCookie1 = "QCCSESSID=jpsim9upk1mpmtf7bu023rfcn7; zg_did=%7B%22did%22%3A%20%2216f308c5a6c120-096498f0382a08-32365f08-100200-16f308c5a6def%22%7D; UM_distinctid=16f308c5b1c1c0-0a9218e3ffdb1e-32365f08-100200-16f308c5b1d321; CNZZDATA1254842228=1218463662-1577066170-https%253A%252F%252Fwww.baidu.com%252F%7C1577066170; hasShow=1; _uab_collina=157706750263744272271734; acw_tc=0e117c2015770675027168129e3a8f4a59719a716592dde533e1b1130b; Hm_lvt_3456bee468c83cc63fb5147f119f1075=1577067503; Hm_lpvt_3456bee468c83cc63fb5147f119f1075=1577067576; zg_de1d1a35bfa24ce29bbf2c7eb17e6c4f=%7B%22sid%22%3A%201577067502200%2C%22updated%22%3A%201577067625196%2C%22info%22%3A%201577067502211%2C%22superProperty%22%3A%20%22%7B%7D%22%2C%22platform%22%3A%20%22%7B%7D%22%2C%22utm%22%3A%20%22%7B%5C%22%24utm_source%5C%22%3A%20%5C%22baidu%5C%22%2C%5C%22%24utm_medium%5C%22%3A%20%5C%22cpc%5C%22%2C%5C%22%24utm_term%5C%22%3A%20%5C%22%E4%BC%81%E6%9F%A5%E6%9F%A5%5C%22%7D%22%2C%22referrerDomain%22%3A%20%22www.baidu.com%22%2C%22cuid%22%3A%20%226c90c17948363c8699eb3d04ca70ea1a%22%7D";
//	String qccCookie2 = "acw_tc=7793462715688754313293767e1233525853b20fe38aa3c02950c7e0d4; zg_did=%7B%22did%22%3A%20%2216d48445955282-0a9526955405dc-39395704-100200-16d484459561ae%22%7D; UM_distinctid=16d48445a2725c-0180c2a2ee5513-39395704-100200-16d48445a2829e; _uab_collina=156887550255987635232936; QCCSESSID=7s3h2gau26tsjfud6d9gj0c577; CNZZDATA1254842228=1236244002-1568872265-https%253A%252F%252Fsp0.baidu.com%252F%7C1570701334; hasShow=1; Hm_lvt_3456bee468c83cc63fb5147f119f1075=1569375066,1569461642,1569588731,1570705029; Hm_lpvt_3456bee468c83cc63fb5147f119f1075=1570705271; zg_de1d1a35bfa24ce29bbf2c7eb17e6c4f=%7B%22sid%22%3A%201570705028400%2C%22updated%22%3A%201570705279131%2C%22info%22%3A%201570705028407%2C%22superProperty%22%3A%20%22%7B%7D%22%2C%22platform%22%3A%20%22%7B%7D%22%2C%22utm%22%3A%20%22%7B%7D%22%2C%22referrerDomain%22%3A%20%22sp0.baidu.com%22%2C%22zs%22%3A%200%2C%22sc%22%3A%200%2C%22cuid%22%3A%20%22997585ecffc553a6d609ca5264a61748%22%7D";
//	String qccCookie3 = "acw_tc=7793462715688754313293767e1233525853b20fe38aa3c02950c7e0d4; zg_did=%7B%22did%22%3A%20%2216d48445955282-0a9526955405dc-39395704-100200-16d484459561ae%22%7D; UM_distinctid=16d48445a2725c-0180c2a2ee5513-39395704-100200-16d48445a2829e; _uab_collina=156887550255987635232936; QCCSESSID=7s3h2gau26tsjfud6d9gj0c577; CNZZDATA1254842228=1236244002-1568872265-https%253A%252F%252Fsp0.baidu.com%252F%7C1570701334; hasShow=1; Hm_lvt_3456bee468c83cc63fb5147f119f1075=1569375066,1569461642,1569588731,1570705029; Hm_lpvt_3456bee468c83cc63fb5147f119f1075=1570705317; zg_de1d1a35bfa24ce29bbf2c7eb17e6c4f=%7B%22sid%22%3A%201570705028400%2C%22updated%22%3A%201570705324116%2C%22info%22%3A%201570705028407%2C%22superProperty%22%3A%20%22%7B%7D%22%2C%22platform%22%3A%20%22%7B%7D%22%2C%22utm%22%3A%20%22%7B%7D%22%2C%22referrerDomain%22%3A%20%22sp0.baidu.com%22%2C%22zs%22%3A%200%2C%22sc%22%3A%200%2C%22cuid%22%3A%20%22a59dc2c2c9f63e0805390b7d08fcecbc%22%7D";
//	String qccCookie4 = "acw_tc=7793462715688754313293767e1233525853b20fe38aa3c02950c7e0d4; zg_did=%7B%22did%22%3A%20%2216d48445955282-0a9526955405dc-39395704-100200-16d484459561ae%22%7D; UM_distinctid=16d48445a2725c-0180c2a2ee5513-39395704-100200-16d48445a2829e; _uab_collina=156887550255987635232936; QCCSESSID=7s3h2gau26tsjfud6d9gj0c577; CNZZDATA1254842228=1236244002-1568872265-https%253A%252F%252Fsp0.baidu.com%252F%7C1570701334; hasShow=1; Hm_lvt_3456bee468c83cc63fb5147f119f1075=1569375066,1569461642,1569588731,1570705029; Hm_lpvt_3456bee468c83cc63fb5147f119f1075=1570705362; zg_de1d1a35bfa24ce29bbf2c7eb17e6c4f=%7B%22sid%22%3A%201570705028400%2C%22updated%22%3A%201570705366433%2C%22info%22%3A%201570705028407%2C%22superProperty%22%3A%20%22%7B%7D%22%2C%22platform%22%3A%20%22%7B%7D%22%2C%22utm%22%3A%20%22%7B%7D%22%2C%22referrerDomain%22%3A%20%22sp0.baidu.com%22%2C%22zs%22%3A%200%2C%22sc%22%3A%200%2C%22cuid%22%3A%20%22bddecaa9374bad787b581d6e8d1a3c25%22%7D";
//	String qccCookie5 = "acw_tc=7793462715688754313293767e1233525853b20fe38aa3c02950c7e0d4; zg_did=%7B%22did%22%3A%20%2216d48445955282-0a9526955405dc-39395704-100200-16d484459561ae%22%7D; UM_distinctid=16d48445a2725c-0180c2a2ee5513-39395704-100200-16d48445a2829e; _uab_collina=156887550255987635232936; QCCSESSID=7s3h2gau26tsjfud6d9gj0c577; CNZZDATA1254842228=1236244002-1568872265-https%253A%252F%252Fsp0.baidu.com%252F%7C1570701334; hasShow=1; Hm_lvt_3456bee468c83cc63fb5147f119f1075=1569375066,1569461642,1569588731,1570705029; Hm_lpvt_3456bee468c83cc63fb5147f119f1075=1570705396; zg_de1d1a35bfa24ce29bbf2c7eb17e6c4f=%7B%22sid%22%3A%201570705028400%2C%22updated%22%3A%201570705402297%2C%22info%22%3A%201570705028407%2C%22superProperty%22%3A%20%22%7B%7D%22%2C%22platform%22%3A%20%22%7B%7D%22%2C%22utm%22%3A%20%22%7B%7D%22%2C%22referrerDomain%22%3A%20%22sp0.baidu.com%22%2C%22zs%22%3A%200%2C%22sc%22%3A%200%2C%22cuid%22%3A%20%222c18ffad6bfcbda6327543372b65ec8f%22%7D";
//	String qccCookie6 = "acw_tc=7793462715688754313293767e1233525853b20fe38aa3c02950c7e0d4; zg_did=%7B%22did%22%3A%20%2216d48445955282-0a9526955405dc-39395704-100200-16d484459561ae%22%7D; UM_distinctid=16d48445a2725c-0180c2a2ee5513-39395704-100200-16d48445a2829e; _uab_collina=156887550255987635232936; QCCSESSID=7s3h2gau26tsjfud6d9gj0c577; CNZZDATA1254842228=1236244002-1568872265-https%253A%252F%252Fsp0.baidu.com%252F%7C1570701334; hasShow=1; Hm_lvt_3456bee468c83cc63fb5147f119f1075=1569375066,1569461642,1569588731,1570705029; Hm_lpvt_3456bee468c83cc63fb5147f119f1075=1570705429; zg_de1d1a35bfa24ce29bbf2c7eb17e6c4f=%7B%22sid%22%3A%201570705028400%2C%22updated%22%3A%201570705433035%2C%22info%22%3A%201570705028407%2C%22superProperty%22%3A%20%22%7B%7D%22%2C%22platform%22%3A%20%22%7B%7D%22%2C%22utm%22%3A%20%22%7B%7D%22%2C%22referrerDomain%22%3A%20%22sp0.baidu.com%22%2C%22zs%22%3A%200%2C%22sc%22%3A%200%2C%22cuid%22%3A%20%22b4f9e2a2ebca725726ba659054863cd6%22%7D";
//	String qccCookie7 = "acw_tc=7793462715688754313293767e1233525853b20fe38aa3c02950c7e0d4; zg_did=%7B%22did%22%3A%20%2216d48445955282-0a9526955405dc-39395704-100200-16d484459561ae%22%7D; UM_distinctid=16d48445a2725c-0180c2a2ee5513-39395704-100200-16d48445a2829e; _uab_collina=156887550255987635232936; QCCSESSID=7s3h2gau26tsjfud6d9gj0c577; CNZZDATA1254842228=1236244002-1568872265-https%253A%252F%252Fsp0.baidu.com%252F%7C1570701334; hasShow=1; Hm_lvt_3456bee468c83cc63fb5147f119f1075=1569375066,1569461642,1569588731,1570705029; Hm_lpvt_3456bee468c83cc63fb5147f119f1075=1570705462; zg_de1d1a35bfa24ce29bbf2c7eb17e6c4f=%7B%22sid%22%3A%201570705028400%2C%22updated%22%3A%201570705466441%2C%22info%22%3A%201570705028407%2C%22superProperty%22%3A%20%22%7B%7D%22%2C%22platform%22%3A%20%22%7B%7D%22%2C%22utm%22%3A%20%22%7B%7D%22%2C%22referrerDomain%22%3A%20%22sp0.baidu.com%22%2C%22zs%22%3A%200%2C%22sc%22%3A%200%2C%22cuid%22%3A%20%223c73a3807a1c39f40a0e283b93cac4ba%22%7D";
//	String qccCookie8 = "acw_tc=7793462715688754313293767e1233525853b20fe38aa3c02950c7e0d4; zg_did=%7B%22did%22%3A%20%2216d48445955282-0a9526955405dc-39395704-100200-16d484459561ae%22%7D; UM_distinctid=16d48445a2725c-0180c2a2ee5513-39395704-100200-16d48445a2829e; _uab_collina=156887550255987635232936; QCCSESSID=7s3h2gau26tsjfud6d9gj0c577; CNZZDATA1254842228=1236244002-1568872265-https%253A%252F%252Fsp0.baidu.com%252F%7C1570701334; hasShow=1; Hm_lvt_3456bee468c83cc63fb5147f119f1075=1569375066,1569461642,1569588731,1570705029; Hm_lpvt_3456bee468c83cc63fb5147f119f1075=1570705495; zg_de1d1a35bfa24ce29bbf2c7eb17e6c4f=%7B%22sid%22%3A%201570705028400%2C%22updated%22%3A%201570705502301%2C%22info%22%3A%201570705028407%2C%22superProperty%22%3A%20%22%7B%7D%22%2C%22platform%22%3A%20%22%7B%7D%22%2C%22utm%22%3A%20%22%7B%7D%22%2C%22referrerDomain%22%3A%20%22sp0.baidu.com%22%2C%22zs%22%3A%200%2C%22sc%22%3A%200%2C%22cuid%22%3A%20%22ba9dfb8068877bae4eab38c4c3e34e28%22%7D";
//	String qccCookie9 = "acw_tc=7793462715688754313293767e1233525853b20fe38aa3c02950c7e0d4; zg_did=%7B%22did%22%3A%20%2216d48445955282-0a9526955405dc-39395704-100200-16d484459561ae%22%7D; UM_distinctid=16d48445a2725c-0180c2a2ee5513-39395704-100200-16d48445a2829e; _uab_collina=156887550255987635232936; QCCSESSID=4bj20k7371sjoi53lkpntgr453; CNZZDATA1254842228=1236244002-1568872265-https%253A%252F%252Fsp0.baidu.com%252F%7C1569459642; hasShow=1; Hm_lvt_3456bee468c83cc63fb5147f119f1075=1569288746,1569288846,1569375066,1569461642; Hm_lpvt_3456bee468c83cc63fb5147f119f1075=1569461915; zg_de1d1a35bfa24ce29bbf2c7eb17e6c4f=%7B%22sid%22%3A%201569461641456%2C%22updated%22%3A%201569461923902%2C%22info%22%3A%201568875501927%2C%22superProperty%22%3A%20%22%7B%7D%22%2C%22platform%22%3A%20%22%7B%7D%22%2C%22utm%22%3A%20%22%7B%7D%22%2C%22referrerDomain%22%3A%20%22sp0.baidu.com%22%2C%22zs%22%3A%200%2C%22sc%22%3A%200%2C%22cuid%22%3A%20%22b66b9124f8c9c6cc99f374d54e27b303%22%7D";
//	String qccCookie10 = "acw_tc=7793462715688754313293767e1233525853b20fe38aa3c02950c7e0d4; zg_did=%7B%22did%22%3A%20%2216d48445955282-0a9526955405dc-39395704-100200-16d484459561ae%22%7D; UM_distinctid=16d48445a2725c-0180c2a2ee5513-39395704-100200-16d48445a2829e; _uab_collina=156887550255987635232936; QCCSESSID=4bj20k7371sjoi53lkpntgr453; CNZZDATA1254842228=1236244002-1568872265-https%253A%252F%252Fsp0.baidu.com%252F%7C1569459642; hasShow=1; Hm_lvt_3456bee468c83cc63fb5147f119f1075=1569288746,1569288846,1569375066,1569461642; Hm_lpvt_3456bee468c83cc63fb5147f119f1075=1569461946; zg_de1d1a35bfa24ce29bbf2c7eb17e6c4f=%7B%22sid%22%3A%201569461641456%2C%22updated%22%3A%201569461952824%2C%22info%22%3A%201568875501927%2C%22superProperty%22%3A%20%22%7B%7D%22%2C%22platform%22%3A%20%22%7B%7D%22%2C%22utm%22%3A%20%22%7B%7D%22%2C%22referrerDomain%22%3A%20%22sp0.baidu.com%22%2C%22zs%22%3A%200%2C%22sc%22%3A%200%2C%22cuid%22%3A%20%2268c483fa81284ffe9e79164c3b8f22fc%22%7D";
//	String qccCookie11 = "acw_tc=779346cb15363053200497407e9088f3dc3b5ec601a0d24982ef81a9ad; UM_distinctid=165b2eeeb621c7-0bf1acb17cf62-541a301f-100200-165b2eeeb63806; _uab_collina=153630532734945961698153; hasShow=1; _umdata=535523100CBE37C3D4598AE14C5C693B791807A4E4EB241F4BFB5F2B7AF2BE2B1DB7B5012EEDB133CD43AD3E795C914C279A299523AE0B38489548802F1D7A6C; PHPSESSID=svjna9cqpfejfajvcqllln4c50; zg_did=%7B%22did%22%3A%20%22165b2eeeb17db-002a293e29c29-541a301f-100200-165b2eeeb18f9%22%7D; CNZZDATA1254842228=1954999719-1536302522-%252F%252Fwww.qichacha.com%252F%7C1536734872; Hm_lvt_3456bee468c83cc63fb5147f119f1075=1536305327,1536542796,1536549836; Hm_lpvt_3456bee468c83cc63fb5147f119f1075=1536736011; zg_de1d1a35bfa24ce29bbf2c7eb17e6c4f=%7B%22sid%22%3A%201536735481405%2C%22updated%22%3A%201536736018987%2C%22info%22%3A%201536305326884%2C%22superProperty%22%3A%20%22%7B%7D%22%2C%22platform%22%3A%20%22%7B%7D%22%2C%22utm%22%3A%20%22%7B%7D%22%2C%22referrerDomain%22%3A%20%22www.qichacha.com%22%2C%22cuid%22%3A%20%222e6c5846a67e5cf5c2e7689b863a270a%22%7D";
//	String qccCookie12 = "acw_tc=779346cb15363053200497407e9088f3dc3b5ec601a0d24982ef81a9ad; UM_distinctid=165b2eeeb621c7-0bf1acb17cf62-541a301f-100200-165b2eeeb63806; _uab_collina=153630532734945961698153; hasShow=1; _umdata=535523100CBE37C3D4598AE14C5C693B791807A4E4EB241F4BFB5F2B7AF2BE2B1DB7B5012EEDB133CD43AD3E795C914C279A299523AE0B38489548802F1D7A6C; PHPSESSID=svjna9cqpfejfajvcqllln4c50; zg_did=%7B%22did%22%3A%20%22165b2eeeb17db-002a293e29c29-541a301f-100200-165b2eeeb18f9%22%7D; zg_de1d1a35bfa24ce29bbf2c7eb17e6c4f=%7B%22sid%22%3A%201536735481405%2C%22updated%22%3A%201536736044068%2C%22info%22%3A%201536305326884%2C%22superProperty%22%3A%20%22%7B%7D%22%2C%22platform%22%3A%20%22%7B%7D%22%2C%22utm%22%3A%20%22%7B%7D%22%2C%22referrerDomain%22%3A%20%22www.qichacha.com%22%2C%22cuid%22%3A%20%22ef599f37586c9f03836622c04ccbf85d%22%7D; CNZZDATA1254842228=1954999719-1536302522-%252F%252Fwww.qichacha.com%252F%7C1536734872; Hm_lvt_3456bee468c83cc63fb5147f119f1075=1536305327,1536542796,1536549836; Hm_lpvt_3456bee468c83cc63fb5147f119f1075=1536736044";
//	String qccCookie13 = "acw_tc=779346cb15363053200497407e9088f3dc3b5ec601a0d24982ef81a9ad; UM_distinctid=165b2eeeb621c7-0bf1acb17cf62-541a301f-100200-165b2eeeb63806; _uab_collina=153630532734945961698153; hasShow=1; _umdata=535523100CBE37C3D4598AE14C5C693B791807A4E4EB241F4BFB5F2B7AF2BE2B1DB7B5012EEDB133CD43AD3E795C914C279A299523AE0B38489548802F1D7A6C; PHPSESSID=svjna9cqpfejfajvcqllln4c50; zg_did=%7B%22did%22%3A%20%22165b2eeeb17db-002a293e29c29-541a301f-100200-165b2eeeb18f9%22%7D; zg_de1d1a35bfa24ce29bbf2c7eb17e6c4f=%7B%22sid%22%3A%201536735481405%2C%22updated%22%3A%201536736077408%2C%22info%22%3A%201536305326884%2C%22superProperty%22%3A%20%22%7B%7D%22%2C%22platform%22%3A%20%22%7B%7D%22%2C%22utm%22%3A%20%22%7B%7D%22%2C%22referrerDomain%22%3A%20%22www.qichacha.com%22%2C%22cuid%22%3A%20%224493157a391821ab547da627caa684c8%22%7D; CNZZDATA1254842228=1954999719-1536302522-%252F%252Fwww.qichacha.com%252F%7C1536734872; Hm_lvt_3456bee468c83cc63fb5147f119f1075=1536305327,1536542796,1536549836; Hm_lpvt_3456bee468c83cc63fb5147f119f1075=1536736078";
//	String qccCookie14 = "acw_tc=779346cb15363053200497407e9088f3dc3b5ec601a0d24982ef81a9ad; UM_distinctid=165b2eeeb621c7-0bf1acb17cf62-541a301f-100200-165b2eeeb63806; _uab_collina=153630532734945961698153; hasShow=1; _umdata=535523100CBE37C3D4598AE14C5C693B791807A4E4EB241F4BFB5F2B7AF2BE2B1DB7B5012EEDB133CD43AD3E795C914C279A299523AE0B38489548802F1D7A6C; PHPSESSID=svjna9cqpfejfajvcqllln4c50; zg_did=%7B%22did%22%3A%20%22165b2eeeb17db-002a293e29c29-541a301f-100200-165b2eeeb18f9%22%7D; zg_de1d1a35bfa24ce29bbf2c7eb17e6c4f=%7B%22sid%22%3A%201536735481405%2C%22updated%22%3A%201536736119057%2C%22info%22%3A%201536305326884%2C%22superProperty%22%3A%20%22%7B%7D%22%2C%22platform%22%3A%20%22%7B%7D%22%2C%22utm%22%3A%20%22%7B%7D%22%2C%22referrerDomain%22%3A%20%22www.qichacha.com%22%2C%22cuid%22%3A%20%225e87d634d5e76c72f4c501bae6d82bcf%22%7D; CNZZDATA1254842228=1954999719-1536302522-%252F%252Fwww.qichacha.com%252F%7C1536734872; Hm_lvt_3456bee468c83cc63fb5147f119f1075=1536305327,1536542796,1536549836; Hm_lpvt_3456bee468c83cc63fb5147f119f1075=1536736119";
//	String qccCookie15 = "acw_tc=779346cb15363053200497407e9088f3dc3b5ec601a0d24982ef81a9ad; UM_distinctid=165b2eeeb621c7-0bf1acb17cf62-541a301f-100200-165b2eeeb63806; _uab_collina=153630532734945961698153; hasShow=1; _umdata=535523100CBE37C3D4598AE14C5C693B791807A4E4EB241F4BFB5F2B7AF2BE2B1DB7B5012EEDB133CD43AD3E795C914C279A299523AE0B38489548802F1D7A6C; PHPSESSID=svjna9cqpfejfajvcqllln4c50; zg_did=%7B%22did%22%3A%20%22165b2eeeb17db-002a293e29c29-541a301f-100200-165b2eeeb18f9%22%7D; zg_de1d1a35bfa24ce29bbf2c7eb17e6c4f=%7B%22sid%22%3A%201536735481405%2C%22updated%22%3A%201536736164812%2C%22info%22%3A%201536305326884%2C%22superProperty%22%3A%20%22%7B%7D%22%2C%22platform%22%3A%20%22%7B%7D%22%2C%22utm%22%3A%20%22%7B%7D%22%2C%22referrerDomain%22%3A%20%22www.qichacha.com%22%2C%22cuid%22%3A%20%22d9a937c13f0f69d71f8dcf8c855b256e%22%7D; CNZZDATA1254842228=1954999719-1536302522-%252F%252Fwww.qichacha.com%252F%7C1536734872; Hm_lvt_3456bee468c83cc63fb5147f119f1075=1536305327,1536542796,1536549836; Hm_lpvt_3456bee468c83cc63fb5147f119f1075=1536736165";
//	String qccCookie16 = "acw_tc=779346cb15363053200497407e9088f3dc3b5ec601a0d24982ef81a9ad; UM_distinctid=165b2eeeb621c7-0bf1acb17cf62-541a301f-100200-165b2eeeb63806; _uab_collina=153630532734945961698153; hasShow=1; _umdata=535523100CBE37C3D4598AE14C5C693B791807A4E4EB241F4BFB5F2B7AF2BE2B1DB7B5012EEDB133CD43AD3E795C914C279A299523AE0B38489548802F1D7A6C; PHPSESSID=svjna9cqpfejfajvcqllln4c50; zg_did=%7B%22did%22%3A%20%22165b2eeeb17db-002a293e29c29-541a301f-100200-165b2eeeb18f9%22%7D; zg_de1d1a35bfa24ce29bbf2c7eb17e6c4f=%7B%22sid%22%3A%201536735481405%2C%22updated%22%3A%201536736203412%2C%22info%22%3A%201536305326884%2C%22superProperty%22%3A%20%22%7B%7D%22%2C%22platform%22%3A%20%22%7B%7D%22%2C%22utm%22%3A%20%22%7B%7D%22%2C%22referrerDomain%22%3A%20%22www.qichacha.com%22%2C%22cuid%22%3A%20%2267bf555e93ab91bda15489cdcb260b7c%22%7D; CNZZDATA1254842228=1954999719-1536302522-%252F%252Fwww.qichacha.com%252F%7C1536734872; Hm_lvt_3456bee468c83cc63fb5147f119f1075=1536305327,1536542796,1536549836; Hm_lpvt_3456bee468c83cc63fb5147f119f1075=1536736204";


	//爬取所需字段,并插入数据库
	public void QccSpiderToDB() throws Exception{
		
		//在数据库查询10000条未爬取的公司进行爬取
			List<CompBasic> compBasicList = qccService.queryCompBasic(1);
			List<CompBasic> list1  = new ArrayList<CompBasic>();
//			List<CompBasic> list2  = new ArrayList<CompBasic>();
//			List<CompBasic> list3  = new ArrayList<CompBasic>();
//			List<CompBasic> list4  = new ArrayList<CompBasic>();
//			List<CompBasic> list5  = new ArrayList<CompBasic>();
//			List<CompBasic> list6  = new ArrayList<CompBasic>();
//			List<CompBasic> list7  = new ArrayList<CompBasic>();
//			List<CompBasic> list8  = new ArrayList<CompBasic>();
//			List<CompBasic> list9  = new ArrayList<CompBasic>();
//			List<CompBasic> list10  = new ArrayList<CompBasic>();
//			List<CompBasic> list11  = new ArrayList<CompBasic>();
//			List<CompBasic> list12  = new ArrayList<CompBasic>();
//			List<CompBasic> list13  = new ArrayList<CompBasic>();
//			List<CompBasic> list14  = new ArrayList<CompBasic>();
//			List<CompBasic> list15  = new ArrayList<CompBasic>();
//			List<CompBasic> list16  = new ArrayList<CompBasic>();
			
			List<List<CompBasic>> createList = createList(compBasicList, 1);
			list1 = createList.get(0);
//			list2 = createList.get(1);
//			list3 = createList.get(2);
//			list4 = createList.get(3);
//			list5 = createList.get(4);
//			list6 = createList.get(5);
//			list7 = createList.get(6);
//			list8 = createList.get(7);
//			list9 = createList.get(8);
//			list10 = createList.get(9);
//			list11 = createList.get(10);
//			list12 = createList.get(11);
//			list13 = createList.get(12);
//			list14 = createList.get(13);
//			list15 = createList.get(14);
//			list16 = createList.get(15);
			
			Runnable qccRunnable1 = new QccRunnable(qccCookie1,list1);
//			Runnable qccRunnable2 = new QccRunnable(qccCookie2,list2);
//			Runnable qccRunnable3 = new QccRunnable(qccCookie3,list3);
//			Runnable qccRunnable4 = new QccRunnable(qccCookie4,list4);
//			Runnable qccRunnable5 = new QccRunnable(qccCookie5,list5);
//			Runnable qccRunnable6 = new QccRunnable(qccCookie6,list6);
//			Runnable qccRunnable7 = new QccRunnable(qccCookie7,list7);
//			Runnable qccRunnable8 = new QccRunnable(qccCookie8,list8);
//			Runnable qccRunnable9 = new QccRunnable(qccCookie9,list9);
//			Runnable qccRunnable10 = new QccRunnable(qccCookie10,list10);
//			Runnable qccRunnable11 = new QccRunnable(qccCookie11,list11);
//			Runnable qccRunnable12 = new QccRunnable(qccCookie12,list12);
//			Runnable qccRunnable13 = new QccRunnable(qccCookie13,list13);
//			Runnable qccRunnable14 = new QccRunnable(qccCookie14,list14);
//			Runnable qccRunnable15 = new QccRunnable(qccCookie15,list15);
//			Runnable qccRunnable16 = new QccRunnable(qccCookie16,list16);
			Thread thread1 = new Thread(qccRunnable1);
//			Thread thread2 = new Thread(qccRunnable2);
//			Thread thread3 = new Thread(qccRunnable3);
//			Thread thread4 = new Thread(qccRunnable4);
//			Thread thread5 = new Thread(qccRunnable5);
//			Thread thread6 = new Thread(qccRunnable6);
//			Thread thread7 = new Thread(qccRunnable7);
//			Thread thread8 = new Thread(qccRunnable8);
//			Thread thread9 = new Thread(qccRunnable9);
//			Thread thread10 = new Thread(qccRunnable10);
//			Thread thread11 = new Thread(qccRunnable11);
//			Thread thread12 = new Thread(qccRunnable12);
//			Thread thread13 = new Thread(qccRunnable13);
//			Thread thread14 = new Thread(qccRunnable14);
//			Thread thread15 = new Thread(qccRunnable15);
//			Thread thread16 = new Thread(qccRunnable16);
//			String name = thread1.getName();
//			System.out.println(name);
//			String name2 = thread2.getName();
//			System.out.println(name2);
//			String name3 = thread3.getName();
//			System.out.println(name3);
//			String name4 = thread4.getName();
//			System.out.println(name4);
//			String name5 = thread5.getName();
//			System.out.println(name5);
//			String name6 = thread6.getName();
//			System.out.println(name6);
//			String name7 = thread7.getName();
//			System.out.println(name7);
//			String name8 = thread8.getName();
//			System.out.println(name8);
//			String name9 = thread9.getName();
//			System.out.println(name9);
//			String name10 = thread10.getName();
//			System.out.println(name10);
			thread1.start();
//			thread2.start();
//			thread3.start();
//			thread4.start();
//			thread5.start();
//			thread6.start();
//			thread7.start();
//			thread8.start();
//			thread9.start();
//			thread10.start();
//			thread11.start();
//			thread12.start();
//			thread13.start();
//			thread14.start();
//			thread15.start();
//			thread16.start();
	}
	
    public static <T> List<List<T>>  createList(List<T> targe,int size) {  
        List<List<T>> listArr = new ArrayList<List<T>>();  
        //获取被拆分的数组个数  
        int arrSize = targe.size()%size==0?targe.size()/size:targe.size()/size+1;  
        for(int i=0;i<arrSize;i++) {  
            List<T>  sub = new ArrayList<T>();  
            //把指定索引数据放入到list中  
            for(int j=i*size;j<=size*(i+1)-1;j++) {  
                if(j<=targe.size()-1) {  
                    sub.add(targe.get(j));  
                }  
            }  
            listArr.add(sub);  
        }  
        return listArr;  
    }  
	
}




//===============================================================================

/*package io.github.biezhi.elves.spider;

import java.net.InetSocketAddress;
import java.net.Proxy;
import java.net.SocketTimeoutException;
import java.util.List;
import java.util.ListIterator;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

import io.github.biezhi.elves.common.Constants;
import io.github.biezhi.elves.ok.http.FastHttpClient;
import io.github.biezhi.elves.qcc.model.CompBasic;
import io.github.biezhi.elves.qcc.model.CompQccCrawl;
import io.github.biezhi.elves.service.QccService;
import io.github.biezhi.elves.utils.StringUtils;

*//**
 * 爬取企查查
 * @author Administrator
 *
 *//*
@Component
public class QccSpider {

	@Autowired
	private QccService qccService;
	
	private String url = "https://www.qichacha.com/";
	
	String qccCookie = "UM_distinctid=162af0f6b4d42b-05d8915431b7eb-414a0229-100200-162af0f6b4e2d1; _uab_collina=152335546758929474446082; acw_tc=779346cc15355056598595827e88e1f7cdc2ee5157c9622dbb6195b785; hasShow=1; _umdata=2FB0BDB3C12E491DA5ED59AB3225F9EA62D14915C508AA48E971AABFDB05B732C6584785BED3ABF0CD43AD3E795C914C8F638C5F624BDAE6B05AC6E095F2B947; PHPSESSID=ock579qm4nvieaou60hi1disc2; zg_did=%7B%22did%22%3A%20%22162af0f6c0c93-00d2e2e25c85e3-414a0229-100200-162af0f6c0d2b2%22%7D; zg_de1d1a35bfa24ce29bbf2c7eb17e6c4f=%7B%22sid%22%3A%201535699110466%2C%22updated%22%3A%201535699876866%2C%22info%22%3A%201535505680977%2C%22superProperty%22%3A%20%22%7B%7D%22%2C%22platform%22%3A%20%22%7B%7D%22%2C%22utm%22%3A%20%22%7B%7D%22%2C%22referrerDomain%22%3A%20%22www.baidu.com%22%2C%22cuid%22%3A%20%2210247c8ebeed9639de060d29b21c4ffe%22%7D; CNZZDATA1254842228=1465212323-1523352163-https%253A%252F%252Fwww.baidu.com%252F%7C1535694812; Hm_lvt_3456bee468c83cc63fb5147f119f1075=1535699178,1535699632,1535699817,1535699877; Hm_lpvt_3456bee468c83cc63fb5147f119f1075=1535699877";

	//爬取所需字段,并插入数据库
	public void QccSpiderToDB(){
		
		//在数据库查询1000条未爬取的公司进行爬取
		while (true) {
			List<CompBasic> compBasicList = qccService.queryCompBasic(1000);
			for (CompBasic compBasic : compBasicList) {
				CompQccCrawl compQccCrawl = new CompQccCrawl();
				//获取企业id
				Long compId = compBasic.getCompId();
				//获取企业名称
				String compName = compBasic.getCompName();
				//获取企业详细url
				String htmlContentByCompName = getHtmlContentByCompName(compName);
				String compDetailUrl = getCompDetailUrl(htmlContentByCompName, compName);
				
				//获取企查查信息id
				String qccId = "";
				//获取企业详情html
				String compDetailHtml = "";
				//企业图谱json数据
				String companyBusinessmap = "";
				//投资族谱json数据
				String companyRelation = "";
				//关联图谱json数据
				String companyMuhou = "";
				//股权结构图json数据
				String companyGuquan = "";
				if (compDetailUrl != "") {
					qccId = compDetailUrl.substring(compDetailUrl.indexOf("_")+1, compDetailUrl.indexOf("."));
					compDetailHtml = getHtmlContentByCompDetailUrl(compDetailUrl);
					companyBusinessmap = getHtmlContentByParamAndKeyNo("cms_businessmap", qccId);
					companyRelation = getHtmlContentByParamAndKeyNo("cms_map", qccId);
					companyMuhou = getHtmlContentByParamAndKeyNo("company_muhouAction", qccId);
					companyGuquan = getHtmlContentByParamAndKeyNo("cms_guquanmap2", qccId);
				}
				
				compQccCrawl.setCompId(compId);
				compQccCrawl.setCompName(compName);
				compQccCrawl.setQccId(qccId);
				compQccCrawl.setCompDetailHtml(compDetailHtml);
				compQccCrawl.setCompanyBusinessmap(companyBusinessmap);
				compQccCrawl.setCompanyRelation(companyRelation);
				compQccCrawl.setCompanyMuhou(companyMuhou);
				compQccCrawl.setCompanyGuquan(companyGuquan);
				
				//将compQccCrawl插入到数据库
				qccService.insertSpiderData(compQccCrawl);
				//将表字段更新为已爬取
				qccService.updateSpiderState(compId);
			}
		}


		CompQccCrawl compQccCrawl = new CompQccCrawl();
		//获取企业id
		Long compId = 152710156689591281L;
		//获取企业名称
		String compName = "广州汽车集团股份有限公司";
		//获取企业详细url
		String htmlContentByCompName = getHtmlContentByCompName(compName);
		String compDetailUrl = getCompDetailUrl(htmlContentByCompName, compName);
		//获取企查查信息id
		String qccId = compDetailUrl.substring(compDetailUrl.indexOf("_")+1, compDetailUrl.indexOf("."));
		//获取企业详情html
		String compDetailHtml = getHtmlContentByCompDetailUrl(compDetailUrl);
		
		//企业图谱json数据
		String companyBusinessmap = getHtmlContentByParamAndKeyNo("cms_businessmap", qccId);
		//投资族谱json数据
		String companyRelation = getHtmlContentByParamAndKeyNo("cms_map", qccId);
		//关联图谱json数据
		String companyMuhou = getHtmlContentByParamAndKeyNo("company_muhouAction", qccId);
		//股权结构图json数据
		String companyGuquan = getHtmlContentByParamAndKeyNo("cms_guquanmap2", qccId);
		
		compQccCrawl.setCompId(compId);
		compQccCrawl.setCompName(compName);
		compQccCrawl.setQccId(qccId);
		compQccCrawl.setCompDetailHtml(compDetailHtml);
		compQccCrawl.setCompanyBusinessmap(companyBusinessmap);
		compQccCrawl.setCompanyRelation(companyRelation);
		compQccCrawl.setCompanyMuhou(companyMuhou);
		compQccCrawl.setCompanyGuquan(companyGuquan);
		
		//将compQccCrawl插入到数据库
		qccService.insertSpiderData(compQccCrawl);
		//将表字段更新为已爬取
		qccService.updateSpiderState(compId);
	
	
	}
	public static void main(String[] args) {
		String compDetailUrl = "/firm_0a59a30c690d6e45d75ff317f30c07db.html";
		String qccId = compDetailUrl.substring(compDetailUrl.indexOf("_")+1, compDetailUrl.indexOf("."));
		System.out.println(qccId);
	}
	
	//通过get请求获取网页内容
	private String getHtmlContentByCompName(String compName) {
		String result = "";
		sleepTime(3000L);
		try {
			io.github.biezhi.elves.ok.http.Response okHttpResponse;
			Proxy proxy = new Proxy(Proxy.Type.HTTP,new InetSocketAddress(Constants.PROXY_IP, Constants.PROXY_PORT));
			okHttpResponse = FastHttpClient
					.newBuilder()
					.proxy(proxy)
					.build()
					.get()
					.addHeader("cookie", qccCookie)
					.url(url+"search?key="+compName)
					.build()
					.execute();
			System.out.println(url+"search?key="+compName);
			result = okHttpResponse.string();
//			System.out.println(result);
			if (result.contains("<script>window.location.href='https://www.qichacha.com/")
					|| result.contains("user_login_normal")) {
				System.err.println("操作过于频繁，休眠1分钟！");
				sleepTime(60000L);
				result = getHtmlContentByCompName(compName);
			}
			if(!StringUtils.isNullOrEmpty(result)){
				return result;
			}
		} catch (SocketTimeoutException e) {
			System.err.println("===连接超时，重新调用getHtmlContentByCompName(compName)方法===");
			sleepTime(3000L);
			getHtmlContentByCompName(compName);
		} catch (Exception e) {
			e.printStackTrace();
			System.err.println("===其他异常，重新调用getHtmlContentByCompName(compName)方法===");
			sleepTime(3000L);
			getHtmlContentByCompName(compName);
		}
		return result;
	}
	
	//通过get请求和公司详情页面的url获取网页内容
	private String getHtmlContentByCompDetailUrl(String compDetailUrl) {
		String result = "";
		try {
			sleepTime(3000L);
			io.github.biezhi.elves.ok.http.Response okHttpResponse;
			Proxy proxy = new Proxy(Proxy.Type.HTTP,new InetSocketAddress(Constants.PROXY_IP, Constants.PROXY_PORT));
			okHttpResponse = FastHttpClient
					.newBuilder()
					.proxy(proxy)
					.build()
					.get()
					.addHeader("cookie", qccCookie)
					.url(url+compDetailUrl)
					.build()
					.execute();
			
			result = okHttpResponse.string();
			if (result.contains("<script>window.location.href='https://www.qichacha.com/")
					|| result.contains("user_login_normal")) {
				System.err.println("操作过于频繁，休眠1分钟！");
				sleepTime(60000L);
				result = getHtmlContentByCompDetailUrl(compDetailUrl);
			}
			if(!StringUtils.isNullOrEmpty(result)){
				return result;
			}
		} catch (SocketTimeoutException e) {
			System.err.println("===连接超时，重新调用getHtmlContentByCompDetailUrl(compDetailUrl)方法===");
			sleepTime(3000L);
			getHtmlContentByCompDetailUrl(compDetailUrl);
		} catch (Exception e) {
			e.printStackTrace();
			System.err.println("===其他异常，重新调用getHtmlContentByCompDetailUrl(compDetailUrl)方法===");
			sleepTime(3000L);
			getHtmlContentByCompDetailUrl(compDetailUrl);
		}
		return result;
	}
	
	//通过get请求和参数(企业图谱,投资族谱,关联图谱,股权结构图对应的参数)和keyNo获取json数据
	private String getHtmlContentByParamAndKeyNo(String param,String keyNo) {
		String result = "";
		String visitUrl = url+param+"?keyNo="+keyNo;
		sleepTime(3000L);
		try {
			io.github.biezhi.elves.ok.http.Response okHttpResponse;
			Proxy proxy = new Proxy(Proxy.Type.HTTP,new InetSocketAddress(Constants.PROXY_IP, Constants.PROXY_PORT));
			okHttpResponse = FastHttpClient
					.newBuilder()
					.proxy(proxy)
					.build()
					.get()
					.addHeader("cookie", qccCookie)
					.url(visitUrl)
					.build()
					.execute();
			
			result = okHttpResponse.string();
//			System.out.println(result);
			if (result.contains("<script>window.location.href='https://www.qichacha.com/")
					|| result.contains("user_login_normal")) {
				System.err.println("操作过于频繁，休眠1分钟！");
				sleepTime(60000L);
				result = getHtmlContentByParamAndKeyNo(param,keyNo);
			}
			if(!StringUtils.isNullOrEmpty(result)){
				return result;
			}
		} catch (SocketTimeoutException e) {
			System.err.println("===连接超时，重新调用getHtmlContentByParamAndKeyNo(param,keyNo)方法===");
			sleepTime(3000L);
			getHtmlContentByParamAndKeyNo(param,keyNo);
		} catch (Exception e) {
			e.printStackTrace();
			System.err.println("===其他异常，重新调用getHtmlContentByParamAndKeyNo(param,keyNo)方法===");
			sleepTime(3000L);
			getHtmlContentByParamAndKeyNo(param,keyNo);
		}
		return result;
	}
	//休眠
	private void sleepTime(Long sleepTime) {
		try {
			Thread.sleep(sleepTime);
			System.err.println("休眠" + sleepTime + "毫秒后，往下执行");
		} catch (Exception e){}
	}
	
	*//**
	 * 获取公司详情页面的url
	 * @param result 通过getHtmlContentByCompName爬取出来的的页面
	 * @return 公司详情页面的url
	 *//*
	public String getCompDetailUrl(String result,String compName){
		Document doc = Jsoup.parse(result);
		Elements allAElements = doc.select("a.ma_h1");
		ListIterator<Element> htmlAResultIter = allAElements.listIterator();
		while (htmlAResultIter.hasNext()) {
			Element element = (Element) htmlAResultIter.next();
			String text = element.text();
			//如果该内容等于compName,表明需要的就是这家公司，获取该a标签下的href
			if (text.equals(compName) || text.equals(compName.replace("（", "(").replace("）", ")"))) {
				String compDetailUrl = element.attr("href");
				return compDetailUrl;
			}
		}
		return "";
	}
}*/

